package tk.mybatis.springboot.spider.processor;

import org.apache.http.util.TextUtils;
import org.jsoup.Jsoup;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import tk.mybatis.springboot.spider.entity.ProductEntity;
import tk.mybatis.springboot.spider.utils.UrlConfig;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;

/**
 * https://cn.misumi-ec.com
 * 爬虫 webmagic
 */
public class YPageProcessor implements PageProcessor {

    private Site site = Site.me()
            .setRetryTimes(3) //这个重试一般用处不大
            .setTimeOut(30000)//在使用代理的情况下,这个需要设置,可以考虑调大线程数目
            .setSleepTime(5000)//使用代理了之后,代理会通过切换IP来防止反扒。同时,使用代理本身qps降低了,所以这个可以小一些
            .setCycleRetryTimes(3)//这个重试会换IP重试,是setRetryTimes的上一层的重试,不要怕三次重试解决一切问题。。
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36")
            .setUseGzip(true);
    private int i = 1;

    /**
     * @param page
     */
    @Override
    public void process(Page page) {


        /**
         * misimi首页
         **/
        if (page.getUrl().regex(UrlConfig.MSM_INDEX).match()) {
            String html = page.getHtml().xpath("/html/body/div[1]").get();
            Document doc = Jsoup.parse(html);
            Element cat = doc.getElementsByClass("l-meganav__category").first();
//            List<Element> cat_uls = cat.getElementsByTag("ul");

            List<Element> cat_uls = cat.getElementsByClass("l-meganav__level3List");
            System.out.println(cat_uls.size());
            for (Element cat_ul : cat_uls) {
                List<Element> cat_lis = cat_ul.getElementsByTag("li");
                System.out.println(cat_lis.size());
                for (Element cat_li : cat_lis) {
                    String newUrl = cat_li.select("a").first().attr("href").toString();
                    System.out.println("********************");
                    System.out.println(" ******三级分类******");
                    System.out.println("********************");
                    System.out.println(newUrl);
                    page.addTargetRequest(newUrl);

                }
            }
        }

        /**
         *  三级分类
         *  列表页
         **/
        if (page.getUrl().regex(UrlConfig.MSM_LEVEL3).match() & !page.getUrl().regex(UrlConfig.MSM_LEVEL3_LIST).match()) {

            if (page.getUrl().toString().contains("?Page=")) {
                int pageNo = i;
                String currentUrl = "";
                String newUrl = "";
                String html = page.getHtml().xpath("//*[@id=\"search_pager_lower\"]").get();
                String mainDiv = page.getHtml().xpath("/html/body/div[1]").toString();
//                System.out.println(mainDiv);
                Document mainDoc = Jsoup.parse(mainDiv);
                Element category = mainDoc.getElementsByClass("m-categoryList--l").first();
                Element spec = mainDoc.getElementsByClass("m-listAreaUnit--spec").first();

                if (null != spec) {
                    System.out.println("第" + pageNo + "页");
                    List<Element> productsLink = spec.getElementsByClass("mc-name");
                    System.out.println("当前页面总共："+productsLink.size()+"件商品");
                    for (Element product : productsLink) {
                        String productDetailLink = product.getElementsByTag("a").first().attr("href").toString();
//                        Element deliveryE = product.getElementsByClass("m-priceTable").first();
                        System.out.println("=========商品详情页链接=========");
                        System.out.println(productDetailLink);
                        page.addTargetRequest(productDetailLink);
                    }

                } else {
                    System.out.println("暂无下一页数据");
                    i = 1;
                }

                // 下一页
                if (!StringUtil.isBlank(html)) {
                    Document doc = Jsoup.parse(html);
                    Element nextPage = doc.getElementsByTag("li").last();
                    if (nextPage.text().contains("下一页")) {
                        pageNo++;
                        System.out.println("=========商品列表页链接=========");
                        currentUrl = page.getUrl().toString();
                        newUrl = currentUrl.substring(0, currentUrl.indexOf("Page=") + 5).toString() + "" + pageNo;
                        System.out.println(newUrl);
                        page.addTargetRequest(newUrl);

                    } else {
                        i = 2;
                        return;
                    }
                } else {
                    System.out.println("无下一页");
                }

            } else {
                // 默认从第一页开始
                System.out.println("默认从第一页开始");
                page.addTargetRequest(page.getUrl().toString() + "?Page=1");
            }
        }
        /**
         * 商品详情页
         */
        if (page.getUrl().regex(UrlConfig.MSM_DETAIL).match()
                & !page.getUrl().regex(UrlConfig.MSM_CODELIST).match()
                & !page.getUrl().regex(UrlConfig.MSM_AREA0).match()
                & !page.getUrl().regex(UrlConfig.MSM_AREA1).match()
                & !page.getUrl().regex(UrlConfig.MSM_AREA3).match()
                ) {

            String html = page.getHtml().xpath("/html/body/div[1]").get();
            Document doc = Jsoup.parse(html);
            Element tabContainer = doc.select("#Tab_container").first();
            if (null != tabContainer) {
                List<Element> tabs = tabContainer.select("a");
                // 判断是否包含tab页
                if (tabs.size() > 0) {
                    Element tab_codeList = doc.getElementById("Tab_codeList");
                    Element tab_area0 = doc.getElementById("Tab_wysiwyg_area_0");
                    Element tab_area1 = doc.getElementById("Tab_wysiwyg_area_1");
                    Element tab_area3 = doc.getElementById("Tab_wysiwyg_area_3");

                    if (null != tab_area0) {//尺寸图
                        page.addTargetRequest(tab_area0.attr("href"));
                    }
                    if (null != tab_area1) {//规格表
                        page.addTargetRequest(tab_area1.attr("href"));
                    }
                    if (null != tab_area3) {//概述、特性
                        page.addTargetRequest(tab_area3.attr("href"));
                    }
                    if (null != tab_codeList) {//型号、交期
                        page.addTargetRequest(tab_codeList.attr("href"));
                    }

                }
            }


        }
        if (page.getUrl().regex(UrlConfig.MSM_CODELIST).match()) {
            parseCodeList(page);
        }

    }


    /**
     * 详情页
     */

    public ProductEntity parseDetail(Page page, ProductEntity productEntity) {
        String html = page.getHtml().xpath("/html/body/div[1]").get();
        Document doc = Jsoup.parse(html);
        Element cats = doc.select(".l-breadcrumb").first();
        List<Element> catList = cats.select("a");
        Element brandE = doc.select(".m-hoverBrandBox__link").first();
        Element absE = doc.select(".mc-text").first();
        Element infoTable = doc.select("m-listTable").first();
        try {
            String productBrand = brandE.select("span").text();
            System.out.println("**********商品品牌**********");
            System.out.println(productBrand);
            productEntity.setProductBrand(productBrand);
            if (!TextUtils.isBlank(productBrand)) {
                if (productBrand.contains("[") & productBrand.contains("]")) {
                    String productFrom = productBrand.substring(productBrand.indexOf("[") + 1, productBrand.indexOf("]"));
                    System.out.println("**********产地**********");
                    System.out.println(productFrom);
                    productEntity.setProductFrom(productFrom);
                } else {
                    System.out.println("**********产地**********");
                    System.out.println("日本");
                    productEntity.setProductFrom("日本");
                }
            }

            if (null != absE) {
                String productAbs = absE.text();
                System.out.println("**********商品简介**********");
                System.out.println(productAbs);
                productEntity.setProductDes(productAbs);
            }

            if (null != infoTable) {
                System.out.println("**********基本情报**********");
                productEntity.setBaseInfo(infoTable.html());
            }
            String cat1 = catList.get(1).text();
            System.out.println("**********一级分类**********");
            System.out.println(cat1);
            productEntity.setProductCat1(cat1);
            if (catList.size() >= 3 & null != catList.get(2)) {
                String cat2 = catList.get(2).text();
                System.out.println("**********二级分类**********");
                System.out.println(cat2);
                productEntity.setProductCat2(cat2);
            }
            if (catList.size() >= 4 & null != catList.get(3)) {
                String cat3 = catList.get(3).text();
                System.out.println("**********三级分类**********");
                System.out.println(cat3);
                productEntity.setProductCat3(cat3);
            }
            if (catList.size() >= 5 & null != catList.get(4)) {
                String cat4 = catList.get(4).text();
                System.out.println("**********四级分类**********");
                System.out.println(cat4);
                productEntity.setProductCat4(cat4);

            }
            if (catList.size() >= 6 & null != catList.get(5)) {
                String cat5 = catList.get(5).text();
                System.out.println("**********五级分类**********");
                System.out.println(cat5);
                productEntity.setProductCat5(cat5);
            }

        } catch (Exception e) {
            // 无需打印异常
        }
        String productName = cats.select("strong").first().text();
        System.out.println("**********商品名称**********");
        System.out.println(productName);
        productEntity.setProductName(productName);
        return productEntity;
    }


    /**
     * 型号／交期
     * 一个型号 对应 一条数据
     *
     * @param page
     */
    public void parseCodeList(Page page) {
        String html = page.getHtml().xpath("//*[@id=\"listContents\"]/table").get();

        ProductEntity productEntity = null;

        Document doc2 = Jsoup.parse(html);
        List<Element> trs = doc2.select("tr");
        for (int i = 1; i < trs.size(); i++) {
            productEntity = new ProductEntity();
            parseDetail(page, productEntity);
            parseArea0(page, productEntity);
            parseArea1(page, productEntity);
            parseArea3(page, productEntity);

            String productSpec = trs.get(i).text();
            System.out.println("**********规格型号**********");
            System.out.println(productSpec);
            productEntity.setProductNo(productSpec);
        }
        //整段html
        String codeHtml = page.getHtml().xpath("//*[@id=\"Tab_codeList_contents\"]").get();
        System.out.println("==========型号/交期==========");
//        System.out.println(codeHtml);
        productEntity.setTabCode(codeHtml);
        productEntity.setProductOpt("新增");
        page.putField("productEntity", productEntity);

    }

    /**
     * 尺寸图
     *
     * @param page
     * @param productEntity
     */
    public void parseArea0(Page page, ProductEntity productEntity) {
        //整段html
        String area0Html = page.getHtml().xpath("//*[@id=\"Tab_wysiwyg_area_0_contents\"]").get();
        System.out.println("==========尺寸图==========");
//        System.out.println(area0Html);
        productEntity.setTabArea0(area0Html);
    }

    /**
     * 规格表
     *
     * @param page
     * @param productEntity
     */
    public void parseArea1(Page page, ProductEntity productEntity) {
        //整段html
        String area1Html = page.getHtml().xpath("//*[@id=\"Tab_wysiwyg_area_1_contents\"]").get();
        System.out.println("==========规格表==========");
//        System.out.println(area1Html);
        productEntity.setTabArea1(area1Html);
    }

    /**
     * 概述／特性
     *
     * @param page
     * @param productEntity
     */
    public void parseArea3(Page page, ProductEntity productEntity) {
//整段html
        String area3Html = page.getHtml().xpath("//*[@id=\"Tab_wysiwyg_area_3_contents\"]").get();
        System.out.println("==========概述／特性==========");
//        System.out.println(area3Html);
        productEntity.setTabArea3(area3Html);
    }

    @Override
    public Site getSite() {
        return this.site;
    }

    /**
     * 打包时需要注释掉main方法
     */
//    public static void main(String[] args) {
//        Spider.create(new YPageProcessor())
//                .addPipeline(new YPipeline())
////                .setDownloader(new DungProxyDownloader())
//                .thread(20)
//                .addUrl("https://cn.misumi-ec.com/vona2/el_wire/E1411000000")//固定环
//                .runAsync();
//    }
}